package com.guiji.quartz.domain;

import cn.hutool.http.HttpException;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.wanghaomiao.xpath.exception.NoSuchAxisException;
import cn.wanghaomiao.xpath.exception.NoSuchFunctionException;
import cn.wanghaomiao.xpath.exception.XpathSyntaxErrorException;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.guiji.quartz.task.DataSaveUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.*;
import java.util.*;

/**
 * @program: cms-vue-plus
 * @description:
 * @author: wangxiaowen
 * @create: 2021-11-17 18:05
 **/
public class RttNews {

	private static final Logger log = LoggerFactory.getLogger(RttNews.class);

	public static void mainMethod() throws Exception {
		HttpResponse response = null;
		String doc;
		try {
			response = HttpRequest.get("https://www.rttnews.com/content/forex.aspx")
				.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
				.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36")
				.header("Accept-Encoding", "gzip, deflate, br")
				.header("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2")
				.header("Host", "www.rttnews.com")
				.setConnectionTimeout(15000)
				.execute();
			List<HttpCookie> cookies = response.getCookies();
			doc = response.body();
			Set<String> resUrl = getUrl(doc);
			getContent(resUrl, cookies);
		} catch (Exception e) {
			log.error("https://www.marketwatch.com/ 代理失败1" + e);
			return;
		}


	}

	private static void getContent(Set<String> listContent, List<HttpCookie> cookies) throws InterruptedException, MalformedURLException, URISyntaxException, NoSuchFunctionException, XpathSyntaxErrorException, NoSuchAxisException {
		for (String url : listContent) {
			Thread.sleep(1000);
			log.info("getContent执行到-{},listContent长度-{}", url, listContent.size());
			String doc = null;
			JXDocument jxDocument = null;
			try {
//				url = "https://www.rttnews.com/3243004/iceland-central-bank-hikes-key-rate-by-50-bps.aspx?type=fts";
				if (null != cookies && cookies.size() > 0) {
					CookieManager ccm = new CookieManager();
					CookieStore CookieStore = ccm.getCookieStore();
					for (HttpCookie cookie : cookies) {
						URL domain = new URL("https://www.marketwatch.com");
						CookieStore.add(domain.toURI(), cookie);
					}
					HttpRequest.setCookieManager(ccm);
				}

				HttpResponse response = HttpRequest.get(url)
					.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36")
					.header("Host", "www.rttnews.com")
					.header("Referer", "https://www.rttnews.com/content/forex.aspx")
					.setConnectionTimeout(15000).execute();
				doc = response.body();
				if (null == doc || doc.isBlank() || doc.isEmpty()) {
					log.info("请求详情页为空");
					continue;
				}
				jxDocument = new JXDocument(doc);
			} catch (HttpException e) {
				log.info("请求详情页失败" + e);
				continue;
			}
			String title = null;
			try {
				title = (String) jxDocument.sel("//div[@class='storyHeadline']/h1/text()").get(0);
			} catch (Exception e) {
				log.error("标题错误-------------" + url + "-------------------标题错误");
				continue;

			}

			StringBuffer content = new StringBuffer();
			String contentStr = null;
			try {
				jxDocument.sel("//div[@id='divBody']//p/text()").forEach(s -> {
					content.append(s + "$$$");
				});

				contentStr = content.toString();
			} catch (Exception e) {
				continue;
			}
			List<Object> newImgs = new ArrayList<>();
			String img = null;
			try {
				img = jxDocument.sel("//div[@class='articleLTImg']/img/@data-img-src").get(0).toString();
				if (null != img && !img.isEmpty()) {
					newImgs.add(img);
				}
			} catch (Exception e) {
				log.error("可能没有图片");
				img = "";
			}

			Date date = new Date();

			if (null != contentStr && contentStr.length() > 50000 || contentStr.replaceAll("\\$|[\\s\\p{Zs}]", "").length() < 20 || newImgs.size() > 20) {
				continue;
			}
			try {
				System.out.println("=======================================================");
				System.out.println(url);
				System.out.println(title);
				System.out.println(img);
				System.out.println(contentStr);
				System.out.println("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++");
				DataSaveUtil.saveData(title, contentStr, newImgs, url, date, "https://www.rttnews.com/content/forex.aspx", "Rttnews-Forex");
				log.info(Thread.currentThread().getName() + "-----------------Rttnews-Forex----------------");
			} catch (Exception e) {
				continue;
			}
		}
	}

	private static Set<String> getUrl(String content) throws Exception {
		JXDocument jxDocument = new JXDocument(content);
		Set<String> res = new HashSet<>();
		String url = null;
		int imgSize = jxDocument.sel("//div[@class='headlineNew2']/h2/a/@href").size();
		if (imgSize > 0) {
			String temp = null;
			for (int k = 0; k < imgSize; k++) {
				temp = jxDocument.sel("//div[@class='headlineNew2']/h2/a/@href").get(k).toString();
				url = temp.strip();
				res.add(url);

			}
		}
		return res;
	}

}



